{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "ename": "ImportError",
     "evalue": "No module named 'readScreen'",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mImportError\u001b[0m                               Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-4-e40d08693e93>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0mnumpy\u001b[0m \u001b[0;32mas\u001b[0m \u001b[0mnp\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m----> 2\u001b[0;31m \u001b[0;32mimport\u001b[0m \u001b[0mreadScreen\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      3\u001b[0m \u001b[0;32mimport\u001b[0m \u001b[0magent\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mImportError\u001b[0m: No module named 'readScreen'"
     ],
     "output_type": "error"
    }
   ],
   "source": [
    "import numpy as np\n",
    "import readScreen\n",
    "import agent\n",
    "\n",
    "SIM_FRAMES = 5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {},
   "outputs": [],
   "source": [
    "def getState():\n",
    "    state = np.zeros((5,1))\n",
    "    \n",
    "    state[0,0] = readScreen.getQlength(upper)\n",
    "    state[1,0] = readScreen.getQlength(lower)\n",
    "    state[2,0] = readScreen.getQlength(left)\n",
    "    state[3,0] = readScreen.getQlength(right)\n",
    "    \n",
    "    state[4,0] = agent.getTLSphase()\n",
    "    \n",
    "    return state"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [],
   "source": [
    "def makeMove(state,action):\n",
    "    if action == 1:\n",
    "        agent.setTLSphase((agent.getTLSphase + 1)%8)\n",
    "           \n",
    "    agent.simulateFrames(SIM_FRAMES)\n",
    "    newState = getState()\n",
    "        \n",
    "    return newState"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [],
   "source": [
    "def getReward(state):\n",
    "    qLengths = state[:4]\n",
    "    reward = (-1) * np.sum(qLengths) * np.std(qLengths)\n",
    "    \n",
    "    return reward"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "Using TensorFlow backend.\n"
     ]
    }
   ],
   "source": [
    "from keras.models import Sequential\n",
    "from keras.layers.core import Dense, Dropout, Activation\n",
    "from keras.optimizers import RMSprop"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [],
   "source": [
    "model = Sequential()\n",
    "model.add(Dense(164, kernel_initializer='lecun_uniform', input_shape=(5,)))\n",
    "model.add(Activation('relu'))\n",
    "#model.add(Dropout(0.2)) I'm not using dropout, but maybe you wanna give it a try?\n",
    "\n",
    "model.add(Dense(150, kernel_initializer='lecun_uniform'))\n",
    "model.add(Activation('relu'))\n",
    "#model.add(Dropout(0.2))\n",
    "\n",
    "model.add(Dense(2, kernel_initializer='lecun_uniform'))\n",
    "model.add(Activation('linear')) #linear output so we can have range of real-valued outputs\n",
    "\n",
    "rms = RMSprop()\n",
    "model.compile(loss='mse', optimizer=rms)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'state' is not defined",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-8-745937423b76>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[0;32m----> 1\u001b[0;31m \u001b[0mmodel\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mpredict\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mstate\u001b[0m\u001b[0;34m.\u001b[0m\u001b[0mreshape\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m,\u001b[0m\u001b[0;36m5\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m,\u001b[0m \u001b[0mbatch_size\u001b[0m\u001b[0;34m=\u001b[0m\u001b[0;36m1\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m      2\u001b[0m \u001b[0;31m#just to show an example output; read outputs left to right: up/down/left/right\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'state' is not defined"
     ],
     "output_type": "error"
    }
   ],
   "source": [
    "model.predict(state.reshape(1,5), batch_size=1)\n",
    "#just to show an example output; read outputs left to right: up/down/left/right"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "ename": "NameError",
     "evalue": "name 'initGridPlayer' is not defined",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mNameError\u001b[0m                                 Traceback (most recent call last)",
      "\u001b[0;32m<ipython-input-9-2438feaa4c1c>\u001b[0m in \u001b[0;36m<module>\u001b[0;34m()\u001b[0m\n\u001b[1;32m     10\u001b[0m \u001b[0;32mfor\u001b[0m \u001b[0mi\u001b[0m \u001b[0;32min\u001b[0m \u001b[0mrange\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0mepochs\u001b[0m\u001b[0;34m)\u001b[0m\u001b[0;34m:\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     11\u001b[0m \u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0;32m---> 12\u001b[0;31m     \u001b[0mstate\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0minitGridPlayer\u001b[0m\u001b[0;34m(\u001b[0m\u001b[0;34m)\u001b[0m \u001b[0;31m#using the harder state initialization function\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[0m\u001b[1;32m     13\u001b[0m     \u001b[0mstatus\u001b[0m \u001b[0;34m=\u001b[0m \u001b[0;36m1\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n\u001b[1;32m     14\u001b[0m     \u001b[0;31m#while game still in progress\u001b[0m\u001b[0;34m\u001b[0m\u001b[0;34m\u001b[0m\u001b[0m\n",
      "\u001b[0;31mNameError\u001b[0m: name 'initGridPlayer' is not defined"
     ],
     "output_type": "error"
    }
   ],
   "source": [
    "model.compile(loss='mse', optimizer=rms)#reset weights of neural network\n",
    "epochs = 3000\n",
    "gamma = 0.975\n",
    "epsilon = 1\n",
    "batchSize = 40\n",
    "buffer = 80\n",
    "replay = []\n",
    "#stores tuples of (S, A, R, S')\n",
    "h = 0\n",
    "for i in range(epochs):\n",
    "    \n",
    "    state = getState() #using the harder state initialization function\n",
    "    status = 1\n",
    "    #while game still in progress\n",
    "    while(status < 10000):\n",
    "        #We are in state S\n",
    "        #Let's run our Q function on S to get Q values for all possible actions\n",
    "        qval = model.predict(state.reshape(1,5), batch_size=1)\n",
    "        if (random.random() < epsilon): #choose random action\n",
    "            action = np.random.randint(0,2)\n",
    "        else: #choose best action from Q(s,a) values\n",
    "            action = (np.argmax(qval))\n",
    "        #Take action, observe new state S'\n",
    "        new_state = makeMove(state, action)\n",
    "        #Observe reward\n",
    "        reward = getReward(new_state)\n",
    "        \n",
    "        #Experience replay storage\n",
    "        if (len(replay) < buffer): #if buffer not filled, add to it\n",
    "            replay.append((state, action, reward, new_state))\n",
    "        else: #if buffer full, overwrite old values\n",
    "            if (h < (buffer-1)):\n",
    "                h += 1\n",
    "            else:\n",
    "                h = 0\n",
    "            replay[h] = (state, action, reward, new_state)\n",
    "            #randomly sample our experience replay memory\n",
    "            minibatch = random.sample(replay, batchSize)\n",
    "            X_train = []\n",
    "            y_train = []\n",
    "            for memory in minibatch:\n",
    "                #Get max_Q(S',a)\n",
    "                old_state, action, reward, new_state = memory\n",
    "                old_qval = model.predict(old_state.reshape(1,5), batch_size=1)\n",
    "                newQ = model.predict(new_state.reshape(1,5), batch_size=1)\n",
    "                maxQ = np.max(newQ)\n",
    "                y = np.zeros((1,2))\n",
    "                y[:] = old_qval[:]\n",
    "                update = (reward + (gamma * maxQ))\n",
    "                y[0][action] = update\n",
    "                X_train.append(old_state.reshape(5,))\n",
    "                y_train.append(y.reshape(2,))\n",
    "            \n",
    "            X_train = np.array(X_train)\n",
    "            y_train = np.array(y_train)\n",
    "            print(\"Frame #: %s\" % (i,))\n",
    "            model.fit(X_train, y_train, batch_size=batchSize, nb_epoch=1, verbose=1)\n",
    "            state = new_state\n",
    "        status += 1\n",
    "        clear_output(wait=True)\n",
    "    if epsilon > 0.1: #decrement epsilon over time\n",
    "        epsilon -= (1/epochs)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 1
}
